{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1-1.导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1-2.读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataPath = '../JupyterData/'\n",
    "train = pd.read_csv(dataPath+\"RentListingInquries_FE_train.csv\")\n",
    "test = pd.read_csv(dataPath+\"RentListingInquries_FE_test.csv\")\n",
    "#显示前5行数据\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 特征工程已做好,下面直接进行模型调参"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1-3.初步确立弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "x_train = train.drop([\"interest_level\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#构造一个交叉验证器实例(StratifiedKFold为分层采样，确保训练集，测试集中各类别样本的比例与原始数据集中相同)\n",
    "# kfold = KFold(n_splits=4,random_state=0,shuffle=False)\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义一个训练模型的方法\n",
    "def modelfit(xgbinstance,x_train,y_train,cv_folds=None,early_stopping_rounds=10):\n",
    "    xgb_param=xgbinstance.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3    #3类分类问题\n",
    "    \n",
    "    #直接调用xgboost,转换数据格式\n",
    "    xgtrain = xgb.DMatrix(x_train,label = y_train)\n",
    "    \n",
    "    #交叉验证寻找最佳参数\n",
    "    cvresult = xgb.cv(xgb_param,xgtrain,num_boost_round=xgbinstance.get_params()['n_estimators'], folds =cv_folds,\n",
    "                     metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "    #训练完的结果导出csv文件\n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')  #index_label 存储行标签，标签名\n",
    "    \n",
    "    #最佳参数n_estimators，即为行数\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    xgbinstance.set_params(n_estimators = n_estimators)\n",
    "    xgbinstance.fit(x_train, y_train, eval_metric='mlogloss')\n",
    "    \n",
    "    #预测:\n",
    "    train_predprob = xgbinstance.predict_proba(x_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\",logloss )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train : 0.4884125621360669\n"
     ]
    }
   ],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #初始定义1000个学习器\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',    #多类分类，返回概率\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, x_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEXCAYAAAC3c9OwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcXGWZ9vHfXdX7ku50FrIvJGEJEAOEXRTZBEZBENEovMLMyOA7DI4zjIOvjjKMKDpuKA6ICowLIKLjgKKyqqBsiawhkISQleykk3TSe9/vH89TSaVT3V2ddHVVp67v53PSdZY6dfepTl31PGczd0dERKS7RL4LEBGRwqSAEBGRjBQQIiKSkQJCREQyUkCIiEhGCggREclIASGSxsz+n5l9P991iBQCBcQQY2Y1ZrbMzD6cNq3WzFaY2YVp0+aY2a/MbLOZNZrZK2Z2vZkNj/MvNbNOM2uKw1Iz+3iOaz/FzFbl8jX6I1M97v5Fd//bHL3eMjM7PRfrzoXBer+G2nYpJgqIIcbdm4DLgRvNbFSc/BVgnrvfC2BmJwK/B/4EHOLu9cBZQAfwtrTVPenuNe5eA1wIfMXMjhyc30T6w8xK8l2DFCF31zAEB+AO4C7gFGATMDZt3hPAt/t4/qXAE92mPQN8OG38XGAB0EgInEPT5h0apzXGZc5Nm3cO8AqwDVgNXA1UA81AF9AUh3E9/F7fAX4dn/80MC2L7XEI8BDwFvAacNHe1ANcC/w4Pm8K4MBlwEpgM3AFcAzwYvzdb0p7nWnAo/H92Aj8BKiP834UX6s5vtanstjGy4B/ja/VCpTE8dXxd3kNOC3DtjgeWAsk06adD7wYHx8LzAO2AuuAr/ewTU8BVvUwrw74IbABWA58FkjEeUnga3EbvAFcGbdjSQ/rWgac3sO8jwFL4vt6X+pvBjDgG8B6YEvcRof39H7n+//rUB3yXoCGvXzjYDiwJv4nvCxtejXQCZzSx/MvJS0g4odeI3BQHD8I2A6cAZQCn4r/Ucvi+BLg/8XxU+N/xoPjc9cAJ6fVeVR83OMHTlodd8QPg2PjB+JPgLv7eE414QP8svico+J2Oay/9ZA5IG4BKoAzgRbgl8BoYHz8gHpnXH563F7lwCjgj8A309a92wdhb9s4bfnngYlAJXBw/D3HpdWXMTyB14Ez0sZ/BlwTHz8JXBIf1wDH97COHt8vQjj8L1Ab61gE/E2cdwXhA3pC3N4PsxcBEf+uNsb3sxz4NvDHOO/dwHygnhAWhxK/JPX0fmvo/6AupiHK3TcTvnlWAb9ImzWc0HW4NjXBzL4S90NsN7PPpi17fJzeRGg9/AhYHOd9EPi1uz/k7u3AVwkfUicSvqHWADe4e5u7Pwr8Cpgbn9sOzDSzYe6+2d3/0s9f7xfu/oy7dxACYnYfy78HWObut7t7R3y9nxO6zQainv9w9xZ3f5DwgX6Xu69399XA48CRAO6+JG6vVnffAHwdeGcv6+1tG6d8y91XunszIfjL4+9S6u7L3P31HtZ9F/H9MLNawrfqu9K2x3QzG+nuTe7+VH82hpklY+2fdvdt7r6M0GK4JC5yEXCju6+Kf6c39Gf9aT4C3Obuf3H3VuDTwAlmNiX+DrWElqO5+0J3X5P2++3L+y2RAmKIMrOLCd/cHga+nDZrM6ErY2xqgrt/ysN+iP8hfMNOecrd6z3sgxgDHAZ8Mc4bR+g6SK2ji/DtdXyctzJOS1ke5wG8n/CBtNzM/mBmJ/Tz11ub9ngHIYx6Mxk4LoZdo5k1Ej5cxgxQPevSHjdnGK8BMLPRZna3ma02s63Aj4GRvay3t22csjJt/hLgHwmtnPXxtcb1sO47gQvMrBy4APiLu6de628IrZdXzexZM3tPLzVmMpLQclyeNi39/R+XXne3x/3Rffs0EbrvxscvJTcRuiPXmdmtZjYsLrqv77dECoghyMxGE/pfPwb8HXCRmb0DwN23E/rtL+jPOt19HeFb93vjpDcJH7yp1zRCV8fqOG+imaX//UyK83D3Z939PEI3zC+Be1Iv05+a+mEl8IcYdqmhxt0/Psj1fCmuc5a7DwMuJnR/pHR/vd62ccbnuPud7v72+Dxn9y8H6cu9QvhwPRv4MCEwUvMWu/tcwvb4MnCvmVVn/2uykfAtfXLatJ3vP6GLZ0LavIn9WHe67tunGhjBrr+zb7n70YQvNgcB/xKn9/R+Sz8pIIamm4BfuvtjsVn9KeB78dsicfyvzeyaGCaY2QRgak8rNLMRhB2ZC+Kke4C/MrPTzKwU+GfCjtI/EwJoO/ApMys1s1MIwXK3mZWZ2UfMrC52m2wldI1A+OY9wszqBmg7pPwKOMjMLon1lJrZMWZ26CDXU0vYAd1oZuOJH1hp1gEHpo33to33YGYHm9mp8X1uIbReOjMtG90JXAW8g7APIrWei81sVGyxNMbJPa7HzCrSB0IL9R7g+niI9WTgnwgtptTv9QkzG29m9YQd630p7fY6JbH+y8xsdvydvwg87e7L4vt7XNxu2+P26Ozj/Zb+yvdOEA39G4D3Eb5Z1Xeb/ghwfdr4ccADhA+ARuBl4HpgRJx/KeE/TuoInvWEPurRaes4n7CzcQvwB+JO3zjvsDhtS1zm/Di9DPgtoatrK/As8Pa0591G6CZopOejmL6QNn4KfezYjssdTDjyaUNc/6OEfRf9qofMO6lL0pZfRdoBAIQPxc+mbZP5cXs+T/jAX5W27HnAivhaV2exjZex+07tWYR9RdsIO/J/lWkbpi0/ifBh/utu038c3+8mwheC9/Xw/FPi7999mE7Y1/XjuL1XAp9j11FMJYQW7ibCUUyfJLQ4rIfXWZbhNb4Q511B2OGe+n0nxOmnEY5camLXEWM1fb3fGvo3WNzYIiI5YWZnA7e4++Q+F5aCoi4mERlQZlZpZueYWUnsavs84QAJGWLUgpAhwcxOBn6TaZ6Ho7CkQJhZFaG77BDCfpJfA59w9615LUz6TQEhIiIZqYtJREQyGnIXABs5cqRPmTIl32WIiAwp8+fP3+juo/pecpchFxBTpkxh3rx5+S5DRGRIMbPlfS+1O3UxiYhIRgoIERHJSAEhIiIZKSBERCQjBYSIiGSkgBARkYwUECIiklHRBMSaLc08+dpqvKur74VFRKR4AuKX3/4XTrhrJi0tzfkuRURkSCiagDh6RrgDYtO2t/JciYjI0FA0AVFSGe5n3rxtS54rEREZGoomIJIxIFqaGvtYUkREoIgCorQqBETbdrUgRESyUTQBUV5dD0DbDt3USkQkG0UTEBU1dQB0NKsFISKSjaIJiMqa0ILobNmW50pERIaGogmIqtoQEK6AEBHJStEERHllLV1u0Kp9ECIi2SiagLBEgu1WgbU15bsUEZEhIWcBYWa3mdl6M3u5h/lmZt8ysyVm9qKZHZWrWlJ2WBUJBYSISFZy2YK4Azirl/lnAzPicDlwcw5rAaDFqkh2bM/1y4iI7BdyFhDu/kegtwsfnQf80IOngHozG5uregBak1WUKiBERLKSz30Q44GVaeOr4rQ9mNnlZjbPzOZt2LBhr1+wLVlNWacCQkQkG/kMCMswzTMt6O63uvscd58zatSovX7BjpIaKrp27PXzRUSKST4DYhUwMW18AvBmLl+wo7SaSgWEiEhW8hkQ9wH/Jx7NdDywxd3X5PIFX9+aoMoVECIi2SjJ1YrN7C7gFGCkma0CPg+UArj7LcADwDnAEmAHcFmuakmZOn4sNcubaWvvoKw0Z7+6iMh+IWefku4+t4/5Dvx9rl4/E6usI2FO09ZGGkaMHMyXFhEZcormTGqAZGW4HlPT1k15rkREpPAVVUCUxntCNG/VfalFRPpSVAFRVj0cgJZtCggRkb4UVUBUDGsAoK1pc54rEREpfEUVEFW1IwDo2NGY50pERApfUQVEdV1oQXQ2KyBERPpSVAFRWRv2QaCAEBHpU1EFhCVL2U6F7ionIpKFogoIgO1WTbJN96UWEelL0QVEc6KG0na1IERE+lJ0AdGSrKGsXS0IEZG+FF1ArG8rJ9mmFoSISF+KLiBq6kdSZ7qrnIhIX4ouIDor6hnmTYSLyYqISE+KLiCssoFhtoNtzS35LkVEpKAVX0BUhbOpt721Ic+ViIgUtqILiNKacD2mpsb1ea5ERKSwFV1AlA8LAbFji24aJCLSm6ILiMphowBo3aouJhGR3hRdQNQMDwHR3qQWhIhIb4ouIGqHjwaga4fuKici0puiC4iSqno6SGAKCBGRXhVdQGBGEzUkWnVPCBGR3hRfQABNyWGUtikgRER6U5QB0VIyjAoFhIhIr3IaEGZ2lpm9ZmZLzOyaDPMnm9kjZvaimf3ezCbksp6UlrIGqjoVECIivclZQJhZEvgOcDYwE5hrZjO7LfZV4IfuPgu4DvhSrupJ11E5kuFdjXR16YJ9IiI9yWUL4lhgibsvdfc24G7gvG7LzAQeiY8fyzA/J7x6NMPZxpbtumCfiEhPchkQ44GVaeOr4rR0LwDvj4/PB2rNbET3FZnZ5WY2z8zmbdiw72dAJ4eNJmnOWxvX7PO6RET2V7kMCMswrXufztXAO83sOeCdwGqgY48nud/q7nPcfc6oUaP2ubDyujEAbNv45j6vS0Rkf1WSw3WvAiamjU8AdvtEdvc3gQsAzKwGeL+7b8lhTQBUN4SA2LFZLQgRkZ7ksgXxLDDDzKaaWRnwIeC+9AXMbKSZpWr4NHBbDuvZadiI0NPV1rh2MF5ORGRIyllAuHsHcCXwO2AhcI+7LzCz68zs3LjYKcBrZrYIOAC4Plf1pKsdORaAribdE0JEpCe57GLC3R8AHug27XNpj+8F7s1lDZlYRT1tlGDbdclvEZGeFOWZ1JixyevYtkk7qUVEelKcAQG0WAXjfV2+yxARKVjFGxC1k6ix1nyXISJSsIo2INorRlLvm3W5DRGRHhRtQHj1KEawlcYdakWIiGRStAGRrD2AEuti00Yd6ioikknRBkR5fepyG6vzXImISGEq2oCoaggnyzXrchsiIhkVbUCkLrfRqsttiIhkVLQBUTMitCBeeHVxnisRESlMRRsQVjmcDpIcVqejmEREMinagCCRoDHZQNkOdTGJiGRSvAEBbC0bQ22rLrchIpJJUQdEW/VYRnRtoL2zK9+liIgUnKIOCB82nrFsYt2WHfkuRUSk4BR1QJQOn0i5dbB+rU6WExHprqgDonr0FAC2rn0jv4WIiBSgog6I+rFTAXjwqfl5rkREpPAUdUBUjpgMwIkjW/JciYhI4SnqgKCqgRbKKdmmfRAiIt0Vd0CY0Vg6isoWnSwnItJdcQcEsL1iLPVt63DXneVERNIVfUB01IzjADaxtbkj36WIiBSUog+IRP0ERtPI6k1b8l2KiEhBKfqAKB8xiYQ5m9Yuz3cpIiIFpc+AMLNpZlYeH59iZleZWX02Kzezs8zsNTNbYmbXZJg/ycweM7PnzOxFMzun/7/CvqkfE86FuOfhJwf7pUVEClo2LYifA51mNh34ATAVuLOvJ5lZEvgOcDYwE5hrZjO7LfZZ4B53PxL4EPBf/ah9QNQeEALi5NHNg/3SIiIFLZuA6HL3DuB84Jvu/klgbBbPOxZY4u5L3b0NuBs4r9syDgyLj+uAN7Mre+DY8Ml0kiCxWZfbEBFJl01AtJvZXOCjwK/itNIsnjceWJk2vipOS3ctcLGZrQIeAP4h04rM7HIzm2dm8zZs2JDFS/dDSTmbS8dQu33ZwK5XRGSIyyYgLgNOAK539zfMbCrw4yyeZxmmdT/ZYC5wh7tPAM4BfmRme9Tk7re6+xx3nzNq1KgsXrp/ttdOYVznara1tA/4ukVEhqo+A8LdX3H3q9z9LjMbDtS6+w1ZrHsVMDFtfAJ7diH9DXBPfJ0ngQpgZFaVD6SGaUy1tSxd3zToLy0iUqiyOYrp92Y2zMwagBeA283s61ms+1lghplNNbMywk7o+7otswI4Lb7OoYSAGOA+pL5VjT2YGmvhCz99bLBfWkSkYGXTxVTn7luBC4Db3f1o4PS+nhR3bF8J/A5YSDhaaYGZXWdm58bF/hn4mJm9ANwFXOp5uOZF/cTDALhwiq7qKiKSUpLNMmY2FrgI+Ex/Vu7uDxB2PqdP+1za41eAk/qzzlwoHT0DgI4Ni/NciYhI4cimBXEdoRXwurs/a2YHAvvXJ+mw8bRZGeVblua7EhGRgtFnC8Ldfwb8LG18KfD+XBY16BIJNldMpGH7Srq6nEQi0wFYIiLFJZud1BPM7H/MbL2ZrTOzn5vZhMEobjC92jaaybzJ6kadUS0iAtl1Md1OOPpoHOFEt/vjtP3KwYcdySRbz8LVb+W7FBGRgpBNQIxy99vdvSMOdwADf7ZanjVMPoxS62TVGwvzXYqISEHIJiA2mtnFZpaMw8XAplwXNtjKxoZDXVtWvpjnSkRECkM2AfHXhENc1wJrgAsJl9/Yv4w6lE6SlG96Jd+ViIgUhGwutbHC3c9191HuPtrd30c4aW7/UlrBChvHpPalrN+mE+ZERPb2jnL/NKBVFIjaKUcyM7GcBW9uzXcpIiJ5t7cBsV+eKFA7eTbjbRNLlq3IdykiInm3twEx6NdLGgzlE2YDsG3583muREQk/3o8k9rMtpE5CAyozFlF+TTmCACSGxbkuRARkfzrMSDcvXYwCykINaPZxHDGtyxh/dYWRg+ryHdFIiJ5s7ddTPut0vGzONSW8/QbOqNaRIqbAqKbmm2vc5CtYv7r3W9+JyJSXBQQ3STO+U9KrZMtS57OdykiInmlgOhu0vEAjN3yPBubWvNcjIhI/mRzue9tZra127AyXgL8wMEoclBVNdBcP4NjE6/xjPZDiEgRy6YF8XXgXwiX+p4AXA18D7gbuC13peVPWUcTRycW8czr6/NdiohI3mQTEGe5+3fdfZu7b3X3W4Fz3P2nwPAc15cXyTP+nVpr5vn5f853KSIieZNNQHSZ2UVmlojDRWnz9sszqpl8AgCzuhaydENTnosREcmPbALiI8AlwPo4XAJcbGaVwJU5rC1/6ibSUTOOYxOv8dAr6/JdjYhIXmRzue+l7v5edx8Zh/e6+xJ3b3b3JwajyEFnRsmBJ3NiYgHfeXRRvqsREcmLbI5imhCPWFpvZuvM7OdmNmEwisurGWfSYNuY1r5Ih7uKSFHKpovpduA+YBzhSKb747Q+mdlZZvaamS0xs2syzP+GmT0fh0Vm1tif4nNq2qm4JTgl8TyPLtTRTCJSfLIJiFHufru7d8ThDmBUX08ysyTwHeBsYCYw18xmpi/j7p9099nuPhv4NvCLfv8GuVLVAKXVnJGYxxd+rduQikjxySYgNprZxWaWjMPFwKYsnncssCTuw2gjnDdxXi/LzwXuymK9g8be/glmJlZQ3rqJVZt35LscEZFBlU1A/DVwEbAWWANcCFyWxfPGAyvTxlfFaXsws8nAVODRLNY7eGacCcA77AU++N2n8lyMiMjgyuYophXufq67j3L30e7+PuCCLNad6bakPZ038SHgXnfvzLgis8vNbJ6ZzduwYUMWLz1AxsyCZBlzy/4IQFfX/nnah4hIJnt7sb5/ymKZVcDEtPEJQE/X0P4QvXQvufut7j7H3eeMGtXn7o+BYwbHfIyjbDFNjRs451uPD95ri4jk2d4GRKbWQXfPAjPMbKqZlRFC4L49VmR2MOGSHU/uZS25NesDJLydi6rmM6q2PN/ViIgMmr0NiD77Wty9g3Cm9e+AhcA97r7AzK4zs3PTFp0L3O3uhdl/M3Y2jDyIS2uf5fHFG3mPWhEiUiR6vCe1mW0jcxAYUJnNyt39AeCBbtM+12382mzWlTdmcMRFjH/sC0ywTbzZWJrvikREBkWPLQh3r3X3YRmGWnfvMVj2S0dcCMC/lf6Yt3a0s2jdtjwXJCKSe7qjXDYapsL00zmjdhkNFfDv9y+gUHvEREQGigIiW8ddQWL7Oj7HLfxpySZ+t0BXeRWR/ZsCIlvTToOGaZw3oZnK0iRX3f0cTa0d+a5KRCRnFBDZSiTAHVv1LOfUL6eto4v/uF/XaBKR/ZcCoj8+/gRUj+ZrI+5n3LByfjpvJb99eU2+qxIRyQkFRH+UVUNZDSx7nD9caFSXJfn7O59j2cbt+a5MRGTAKSD66++fgrpJlD52Hb/9xNsx4OwbH2e79keIyH5GAdFfJeVQUgFrnmfiG/dw+2XH0NzeyUk3PEpbR1e+qxMRGTAKiL1x5TNQUQe/vpqTx3Ry/fmH09jcznFffJiOToWEiOwfFBB7www+9hh4F9x8Ih85dhKf/atD2byjnWOuf5iW9oxXLRcRGVIUEHtrxDSonwQ7NsGL9/C3Jx/I5987c2dIbN7elu8KRUT2iQJiX1z1HJQPg19+HBpXcNlJU5k2qpptLR0c98VHmL98c74rFBHZawqIfZFIwhXx8t83nwgdrTzyz6dw35UnYQYX3vxnbvnD67Rrv4SIDEEKiH01fApc+ANo3QbfnAXArAn1PPOZ06mvKuWG37zKrGsf5InFG/Nbp4hIPykgBsJh58PbPwlNa+HG2QDUVZbyl387g1svOZoudy7+wdMced2DrHxrR56LFRHJjgJioJz6b3DIe2DzG/CtowEwM848bAwvfP5MJgyvZPOOdk7+ymMc98WHebOxOc8Fi4j0TgExUBJJuPA2qBgOby2BF3+2c1ZFaZIn/vVUnvr0aYyqKWPd1lZOvOFRpn/mAZasb8pj0SIiPbOhduObOXPm+Lx58/JdRs/am+GrB0PrFnjfLTB77h6LrNq8g/ff/GfWbW0FCPsqLpjFuw4ZRXlJcrArFpEiYGbz3X1Ov56jgMiB1ib4xkxo2QJnfgFOuDKcXNfNpqZW3nvTE6xpbNl58+9RteXcNPdIjpnSQCKx53NERPaGAqKQdLTC1w8NJ9JVj4JPvBCuBptp0c4u/vT6Jv7pp8+zKZ5gZ8CYugpuv+wYDj6gFssQMCIi2VJAFJquLrjxbbBlBZRWwRVPhDOwe7GjrYOHXlnH5/53AVua24EQFg3VZVz97oM54cARTB5RpcAQkX5RQBSqJQ/DnR8Ed7joh3Doe7J62samVj5w859ZubmZzi7f2Q2VCox/PesQjj9wBBMbKhUYItIrBUQha1wB/3UCtDVB7Vi48lkor8366e7O6xu28+TSTXzjoUVs3t5G+jtXV1nKJcdP5vDxdRwyppZJDVXahyEiOykgCl1HKzz0eXj6ZsBg9Ez4v3/eq1W5O0vWN/GxH85jdWNoYXR1eysTBsmEMbauki9dcASTGqoYW1dBSVJHN4sUm4ILCDM7C7gRSALfd/cbMixzEXAt4MAL7v7h3tY5pAMiZcVTcPvZ4XLhb/swvPt6qGrY59W2tHfy2tptvLZ2G19/6DU2NLXRldY1lWIGCTMaqsu49MQpTGqoYmJDFROHV9JQXabuKpH9UEEFhJklgUXAGcAq4Flgrru/krbMDOAe4FR332xmo919fW/r3S8CAqC9Bb59NGxdBYnScDjsnMvCHesG2IZtrSxet43P/PIlVm9upsuhyx139ggPCC0PMyNhcEBtBVedPoOxdRWMratgTF0l1WVJhYjIEFNoAXECcK27vzuOfxrA3b+UtsxXgEXu/v1s17vfBETKmhfhjveEE+uS5fDeb8KsD4YzswfB9tYOVm1uZsVbO/jCr17hzS3NeB8BAmFHucUgMQun5I+pq+SfzzyIkTXl1FWWUldZyrDKUmrLS7Q/RCTPCi0gLgTOcve/jeOXAMe5+5Vpy/yS0Mo4idANda27/7a39e53AQHh6Kalj8FPLwk7sUur4ILvwSF/lfEEu8HU2tHJ+q2trNnSwpotzXzq3hdp6+gimTCcsC+ktyBJZ5YKFsOAzi6nJGlcdeoMhu0MlJLws6J057TykoRaLCL7qNAC4gPAu7sFxLHu/g9py/wKaAcuAiYAjwOHu3tjt3VdDlwOMGnSpKOXL1+ek5rzzh0W3ge/+DvoaIayWph7J0w5Oe9B0Rd3Z0tzOxubWtnY1MaW5na+/JtXWd3YHEIEdgVJHO++U703qRYL7AqYVOBgxri6Cv7h1BlUlyepKiuhpqKE2vLws6a8hOoytWKkuBVaQGTTxXQL8JS73xHHHwGucfdne1rvftmC6K6zA164E+67CvBwBvZZN8DhF0JZVb6rG3BtHV1sbWlna3M7W5rb2drSwX/cv4CVm5tp6wg3WypJ2s6ASQUOWbZcukvlRCpcUrFhBh2dTmnS+IdTZ1BZlqS8NEllHCpKE1SWJiktSVCaTFCSMMpKws/SZJhWVpKgNBmmlyYSCiUpGIUWECWE7qPTgNWEndQfdvcFacucRdhx/VEzGwk8B8x29009rbcoAiKlvSXcqW7zG+GIp0QJHHcFzPnrPs/ILhbuzva2Tna0ddDc1smOtk62t3bQFIevPfgabza24O60d4a/9VT3WKolA+xT4GTL0h6kHqdaUcn0JlL35QndcRCCsqf1tnc6pSUJPnHaDJIJI2lGMmGUJG2P8YQZJYlEGE/E+d0eh/FEt/Hslk0d6CCFo6ACAsDMzgG+Sdi/cJu7X29m1wHz3P0+C39BXwPOAjqB69397t7WWVQBkeIOy/8E93wUdsQ701UMhwu+C9NPH7Qd2sWkvbOLlvZOmts7aWnroqWjk+a2MN7e2RUHp6PTdxtv7+yiraOLts4ubnx4Ee2dvtuHevp/N2fXB38yYbsFVjpPe551mz6U7BEXaS25VFD21uDKtMyu0A2PdtuePdTQkQrbPlp36euqqyzlxOkjd+/aZFd3Z2rlf1y0AYBTDh69e7coFg/q2P15YV27B2rqMPTuB4IYxrsPO4AjJw3vte6eFFxA5EJRBkS6rWtg/h3wxNegsz0cFvuuz4Qjn2rH5Ls6ybOuLqejy+nscjq6uujqgo6urjgepnd/nFq2s/t8dzo705Z1p7Ori47O9HHnB4+/gRMOVli3LVzCPpVoqVZb95aP7/wn6Ojjgx36+PCPr9e5l0HT23K9NYQ8fZkePkrTL5HT43p6L2WnqSOqeOxf3pXl0rtTQBSTjjZ49X74+cfAO8O0qe+EIz4Ah74XKuvzW5+IFBRpxKPtAAAQEElEQVQFRLHasAh+fAFsWUX4LmLhgoBHfABmvBtKK/JdoYjk2d4EREmuipFBNOog+OTLob27+i/w0s/g2e/DwvvD/OrR8J6vw9R3QEVdfmsVkSFDLYj9VVcnvPFHeOleeOGuXd1Q5cPgpKvCzu0xb4OELtwnUgzUxSSZdbbDqmfDfSmevgXatofpiVI47HyYfhpMOxVqRue3ThHJGQWEZKdpA7z+KDz4Gdi+kZ3HUJRVw7F/F1oXE4+FZGleyxSRgaOAkP7r6oK1L4bWxWNf3NUVZUk46KzQuph+GgyfktcyRWTfKCBk37VsDfsuljwMz98JnfG4dix0QZ35BZjydhg2Lq9likj/KCBkYLnDpiWw5JEQGot/G3Z+A5RUwKyLYPLbYdLxUD+p4C8oKFLMFBCSW12dsPYlWPYEPP5VaN68a16yDA55TwiLScfDAYfrEiAiBUQBIYOrqxPWvxJuofr7L8frRKX9PVkS3nF12OE94RidgyGSRwoIyb/GlSEwHvwsbF8frkKbUlYNR18WwmLcbKifrG4pkUGiM6kl/+onhmHWB8J46zZYPT+ExpP/BU/elLawhVbFcX8H4+fA+KOhekReyhaRPakFIYOroxXWLYA3nwvDyz+H9h275pdUhIsNjj86hMbYWeGKtSKyT9TFJENTa1MIi9XzYNU8WPwgdLbtmm8JOPpSGDsbxr4NRs+EkrK8lSsyFKmLSYam8hqYenIYUrasDoGxej7MuwPm3bb7c8qqwy1Yx74NxsyCA2aGaSIyYNSCkKGhqyvcenXNC7DmeZj/39DSuPsyloBDz4UxR+waasdqR7gI6mKSYuMOjStg3cvh/Iynb4Xmt9jtUNtECUw+EQ5IhcbhMPJgdVFJ0VFAiEC4XMi6BSE01r0EL/8C2prSFrBwIt+Yw0NoHBB/VjXkrWSRXFNAiPSkswPeej2ExtqX0rqo0v/+LdyqdfZHYOQMGDEDRkwP16BSN5UMcQoIkf5q2hBaGWtfgrUvw8v37n5yX0pZDRx8Trh736hDYfSh4Qq3upyIDBEKCJGB0NUZ7u+9aTFsXBIuWLhpMSx/Mu3qtlFpVbh/xojpu1odI6dD5fD81C7SAx3mKjIQEkkYPjkM00/ffV7rNtiwCDYshA2vwnM/gVd/laHVYVBeCzPPjaERw2P4FO0glyFDLQiRgdDZDpuXx1bH4vDzuZ/sugFTupIKmPrOGBrTd7U+ag7Qvg7JGXUxiRSi5kbY9HoIjU1LYN7t4XDc7q0OS4ZLi6R2jo+cHh9P00mAss8KrovJzM4CbgSSwPfd/YZu8y8F/hNYHSfd5O7fz2VNIoOush4mHB0GgFM/G352dcHWVSE0Nsb9HC/eA28+z+5HVxHutzHphN33c4yYDnUTtaNcciZnLQgzSwKLgDOAVcCzwFx3fyVtmUuBOe5+ZbbrVQtCikJ7865WRyo8Xv11t/M5otSO8pEzoOFAaJgWfurwXElTaC2IY4El7r4UwMzuBs4DXun1WSICpZXxRL7Dd5/uDts3xP0cS3YFyPpXYOF9uy9ryXCNqlRojJwBIw8KP3XzJslCLgNiPLAybXwVcFyG5d5vZu8gtDY+6e4ruy9gZpcDlwNMmjQpB6WKDBFmoWVQMxqmnLT7vM72cOmRt94IJwW+tTQMi34HHS27L5sohfFHhaOqUkP95PCzdiwkEoPz+0hBy2VAZGrbdu/Puh+4y91bzewK4L+BU/d4kvutwK0QupgGulCR/UKyNOzQHjEN6HZ4bmcHbF4GGxeFYdNieOV+WPkMe/63JFz4sKIOjrgIGqbC8KmhJTJ8su7PUURyGRCrgIlp4xOAN9MXcPdNaaPfA76cw3pEileyJOzYHjkdOCdMO+874WdHG2xZGQIkNTz/E2jeDM98N8O6ysN9xtODI/W4Ytjg/D4yKHIZEM8CM8xsKuEopQ8BH05fwMzGuvuaOHousDCH9YhIJiVlaS2P6Mz/CD/dYcem0G21+Y3YbRV//uVH7NH6SJSG+403HBjDY2rsupoMNWPUdTXE5Cwg3L3DzK4Efkc4zPU2d19gZtcB89z9PuAqMzsX6ADeAi7NVT0ishfMoHpkGCYes+f81m2Zw2PBL/e8LAmEkwQnnQD1k0Jo1KeGSTrqqgDpRDkRyY2O1nB2eeMKaFwWf66AxQ9B23Yy7vsYeVAIi1RopAdJ1QgFyD4otMNcRaSYlZTHq98elHl+a1PY99G4IgZJDJPXH4Elj9DjzvPpZ8TQ6BYklcMVIANMASEi+VFeEy6bPvrQzPNbtu5qdaTCY/NyWPp7WPy7zM8prYID39Wt9RGDRDvQ+00BISKFqWJY5pMFU5o3p7U+0oJk6WPQviPzc8qqYdqpu+/7SA3lNbn7XYYoBYSIDE2Vw8Mw9m17znOHHW/Flsfy3YPk1V9nvikUFgJk+ukZdqJPDGe3FxkFhIjsf8ygekQYxh+15/zUJUvS932kfi76HXQ07/mc1CG8dRNjq2Mi1E3a9Xg/vOKuAkJEik/6JUsyHb7b1QVN63YPj1SYvPbAnpcuCSuNXVjvCsFRNwHqxsOw+LN69JA7D0QBISLSXSIBw8aGYdLxe87v6oKmtdAYj8LasmLX48UPZ26BQDgLffzRMTjiUDc+hMmwCVDVUFBHYikgRET6K5GAYePCMCnDNUhT+0C2roItq2FrHFKPF94XzhPJpKQyrHPYhF2tkFSADPK+EAWEiMhAS98HkmknOoRWyPYNu0Jky6oYIqvCeSBtf2CPc0GGHwifeC7n5acoIERE8iGRgNoDwjD+6MzLdLTBtjW7gmNchh3uOaSAEBEpVCVl4XDb4ZPz8vJDa5e6iIgMGgWEiIhkpIAQEZGMFBAiIpKRAkJERDJSQIiISEYKCBERyUgBISIiGQ25e1Kb2QZg+V4+fSSwcQDLGQxDsWYYmnWr5sGhmgdH95onu/uo/qxgyAXEvjCzef29aXe+DcWaYWjWrZoHh2oeHANRs7qYREQkIwWEiIhkVGwBcWu+C9gLQ7FmGJp1q+bBoZoHxz7XXFT7IEREJHvF1oIQEZEsKSBERCSjogkIMzvLzF4zsyVmdk2+6+mJmS0zs5fM7HkzmxenNZjZQ2a2OP4cnucabzOz9Wb2ctq0jDVa8K243V80s8G9JVbvNV9rZqvjtn7ezM5Jm/fpWPNrZvbuPNU80cweM7OFZrbAzD4Rpxfstu6l5kLf1hVm9oyZvRDr/vc4faqZPR239U/NrCxOL4/jS+L8KQVU8x1m9kbatp4dp/f/78Pd9/sBSAKvAwcCZcALwMx819VDrcuAkd2mfQW4Jj6+Bvhynmt8B3AU8HJfNQLnAL8BDDgeeLqAar4WuDrDsjPj30g5MDX+7STzUPNY4Kj4uBZYFGsr2G3dS82Fvq0NqImPS4Gn4za8B/hQnH4L8PH4+P8Ct8THHwJ+WkA13wFcmGH5fv99FEsL4lhgibsvdfc24G7gvDzX1B/nAf8dH/838L481oK7/xF4q9vknmo8D/ihB08B9WY2dnAq3aWHmntyHnC3u7e6+xvAEsLf0KBy9zXu/pf4eBuwEBhPAW/rXmruSaFsa3f3pjhaGgcHTgXujdO7b+vUe3AvcJqZ2SCVC/Rac0/6/fdRLAExHliZNr6K3v9o88mBB81svpldHqcd4O5rIPwHBEbnrbqe9VRjoW/7K2Nz+7a0rruCqzl2YRxJ+JY4JLZ1t5qhwLe1mSXN7HlgPfAQoTXT6O4dGWrbWXecvwUYMbgV71mzu6e29fVxW3/DzMq71xz1ua2LJSAyJXuhHt97krsfBZwN/L2ZvSPfBe2jQt72NwPTgNnAGuBrcXpB1WxmNcDPgX909629LZphWl7qzlBzwW9rd+9099nABEIr5tBMi8WfBVF395rN7HDg08AhwDFAA/CvcfF+11wsAbEKmJg2PgF4M0+19Mrd34w/1wP/Q/hDXZdqCsaf6/NXYY96qrFgt727r4v/wbqA77Gra6NgajazUsIH7U/c/RdxckFv60w1D4VtneLujcDvCf309WZWEmel17az7ji/juy7MAdcWs1nxW4+d/dW4Hb2YVsXS0A8C8yIRySUEXYq3ZfnmvZgZtVmVpt6DJwJvEyo9aNxsY8C/5ufCnvVU433Af8nHkFxPLAl1T2Sb936X88nbGsINX8oHqkyFZgBPJOH+gz4AbDQ3b+eNqtgt3VPNQ+BbT3KzOrj40rgdML+k8eAC+Ni3bd16j24EHjU457gwdJDza+mfXkwwj6T9G3dv7+Pwd7znq+BsAd/EaFf8TP5rqeHGg8kHNHxArAgVSehb/MRYHH82ZDnOu8idBO0E76V/E1PNRKatd+J2/0lYE4B1fyjWNOL8T/P2LTlPxNrfg04O081v53QBfAi8Hwczinkbd1LzYW+rWcBz8X6XgY+F6cfSAisJcDPgPI4vSKOL4nzDyygmh+N2/pl4MfsOtKp338futSGiIhkVCxdTCIi0k8KCBERyUgBISIiGSkgREQkIwWEiIhkpIAQEZGMFBAiWTCz2d0uUX2uDdBl483sH82saiDWJTKQdB6ESBbM7FLCiUVX5mDdy+K6N/bjOUl37xzoWkTSqQUh+xUzm2LhZjXfizdReTBehiDTstPM7LfxyrmPm9khcfoHzOzleCOWP8bLs1wHfDDegOWDZnapmd0Ul7/DzG62cKOcpWb2znjF0oVmdkfa691sZvNs95u7XAWMAx4zs8fitLkWbhr1spl9Oe35TWZ2nZk9DZxgZjeY2Svxqp1fzc0WlaKWj9PaNWjI1QBMATqA2XH8HuDiHpZ9BJgRHx9HuJ4OhMsQjI+P6+PPS4Gb0p67c5xwg5a7CZcyOA/YChxB+AI2P62W1CUxkoQLq82K48uIN4kihMUKYBRQQrhswvviPAcuSq2LcGkKS69Tg4aBHNSCkP3RG+7+fHw8nxAau4mXoz4R+Fm8nv53CXdDA/gTcIeZfYzwYZ6N+93dCeGyzt1f8nDl0gVpr3+Rmf2FcP2cwwh3U+vuGOD37r7Bw30GfkK4Gx5AJ+EqqRBCqAX4vpldAOzIsk6RrJX0vYjIkNOa9rgTyNTFlCDcDGZ29xnufoWZHQf8FbDznr5ZvmZXt9fvAkrilUqvBo5x982x66kiw3p6uytZi8f9Du7eYWbHAqcRrk58JeHuZyIDRi0IKUoebmLzhpl9AHbe0P1t8fE0d3/a3T8HbCRcQ38b4R7Le2sYsB3YYmYHEG4IlZK+7qeBd5rZSDNLAnOBP3RfWWwB1bn7A8A/Em7EIzKg1IKQYvYR4GYz+yzhfr53Ey61/p9mNoPwbf6ROG0FcE3sjvpSf1/I3V8ws+cIXU5LCd1YKbcCvzGzNe7+LjP7NOE+BAY84O6Z7v9RC/yvmVXE5T7Z35pE+qLDXEVEJCN1MYmISEbqYpL9npl9Bzip2+Qb3f32fNQjMlSoi0lERDJSF5OIiGSkgBARkYwUECIikpECQkREMvr/AUgucUrym+EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c4d3710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>test-mlogloss-mean</th>\n",
       "      <th>test-mlogloss-std</th>\n",
       "      <th>train-mlogloss-mean</th>\n",
       "      <th>train-mlogloss-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>333</td>\n",
       "      <td>0.586414</td>\n",
       "      <td>0.001854</td>\n",
       "      <td>0.459538</td>\n",
       "      <td>0.001172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>334</td>\n",
       "      <td>0.586378</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.459270</td>\n",
       "      <td>0.001183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>335</td>\n",
       "      <td>0.586423</td>\n",
       "      <td>0.001817</td>\n",
       "      <td>0.458977</td>\n",
       "      <td>0.001130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>336</td>\n",
       "      <td>0.586373</td>\n",
       "      <td>0.001819</td>\n",
       "      <td>0.458641</td>\n",
       "      <td>0.001121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>337</td>\n",
       "      <td>0.586325</td>\n",
       "      <td>0.001805</td>\n",
       "      <td>0.458323</td>\n",
       "      <td>0.001151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     n_estimators  test-mlogloss-mean  test-mlogloss-std  train-mlogloss-mean  \\\n",
       "333           333            0.586414           0.001854             0.459538   \n",
       "334           334            0.586378           0.001823             0.459270   \n",
       "335           335            0.586423           0.001817             0.458977   \n",
       "336           336            0.586373           0.001819             0.458641   \n",
       "337           337            0.586325           0.001805             0.458323   \n",
       "\n",
       "     train-mlogloss-std  \n",
       "333            0.001172  \n",
       "334            0.001183  \n",
       "335            0.001130  \n",
       "336            0.001121  \n",
       "337            0.001151  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv(\"1_nestimators.csv\")\n",
    "data.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第一步调参，确立的弱学习器为338个，logloss为0.4884"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2-1.调整树的参数:max_depth&min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 0.7,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.1,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 5,\n",
       " 'min_child_weight': 1,\n",
       " 'missing': None,\n",
       " 'n_estimators': 338,\n",
       " 'nthread': 1,\n",
       " 'objective': 'multi:softprob',\n",
       " 'reg_alpha': 0,\n",
       " 'reg_lambda': 1,\n",
       " 'scale_pos_weight': 1,\n",
       " 'seed': 3,\n",
       " 'silent': 1,\n",
       " 'subsample': 0.5}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb1.get_xgb_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(6, 10, 2), 'min_child_weight': range(1, 5, 2)}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(6,10,2)\n",
    "min_child_weight = range(1,5,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58523, std: 0.00340, params: {'max_depth': 6, 'min_child_weight': 1},\n",
       "  mean: -0.58414, std: 0.00375, params: {'max_depth': 6, 'min_child_weight': 3},\n",
       "  mean: -0.60011, std: 0.00462, params: {'max_depth': 8, 'min_child_weight': 1},\n",
       "  mean: -0.59602, std: 0.00486, params: {'max_depth': 8, 'min_child_weight': 3}],\n",
       " {'max_depth': 6, 'min_child_weight': 3},\n",
       " -0.5841421419938186)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=338,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  粗调后得出的 max_depth=6, min_child_weight=3，logloss为0.5841"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([561.06240001, 559.95180006, 718.72920008, 587.23180003]),\n",
       " 'mean_score_time': array([2.77179999, 1.58599982, 6.16199994, 2.02659998]),\n",
       " 'mean_test_score': array([-0.58522737, -0.58414214, -0.60011087, -0.5960162 ]),\n",
       " 'mean_train_score': array([-0.40773329, -0.4225965 , -0.25178966, -0.28964737]),\n",
       " 'param_max_depth': masked_array(data=[6, 6, 8, 8],\n",
       "              mask=[False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 1, 3],\n",
       "              mask=[False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 6, 'min_child_weight': 1},\n",
       "  {'max_depth': 6, 'min_child_weight': 3},\n",
       "  {'max_depth': 8, 'min_child_weight': 1},\n",
       "  {'max_depth': 8, 'min_child_weight': 3}],\n",
       " 'rank_test_score': array([2, 1, 4, 3]),\n",
       " 'split0_test_score': array([-0.57894534, -0.57827439, -0.59356332, -0.58960506]),\n",
       " 'split0_train_score': array([-0.40884193, -0.42370689, -0.25174664, -0.28990569]),\n",
       " 'split1_test_score': array([-0.58478884, -0.58309554, -0.59778919, -0.59537233]),\n",
       " 'split1_train_score': array([-0.4105807 , -0.4250349 , -0.25184358, -0.28865201]),\n",
       " 'split2_test_score': array([-0.58609023, -0.58293948, -0.5986249 , -0.59335011]),\n",
       " 'split2_train_score': array([-0.40826136, -0.42352707, -0.25287603, -0.29219123]),\n",
       " 'split3_test_score': array([-0.58781875, -0.5878289 , -0.60412592, -0.59756191]),\n",
       " 'split3_train_score': array([-0.40637093, -0.42095584, -0.25352509, -0.29049715]),\n",
       " 'split4_test_score': array([-0.58849469, -0.58857374, -0.60645293, -0.60419411]),\n",
       " 'split4_train_score': array([-0.40461152, -0.41975781, -0.24895693, -0.28699078]),\n",
       " 'std_fit_time': array([ 7.11497179,  6.5265527 ,  6.39326782, 63.05238503]),\n",
       " 'std_score_time': array([0.93467817, 0.46680575, 0.71098046, 0.97939685]),\n",
       " 'std_test_score': array([0.00339975, 0.00374705, 0.0046218 , 0.0048555 ]),\n",
       " 'std_train_score': array([0.00205996, 0.0019388 , 0.0015634 , 0.00174976])}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.584142 using {'max_depth': 6, 'min_child_weight': 3}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt8VeWd7/HPNyHhIl4hVhAstIpClYOaiq3jdY6K1cFaqYrSU5xjmVPl2OkcbfVMW8dLHe3MsVZr7ViLl9YLiqODHVTEY9V6qUSPooIootWIlYii5WIg4Xf+WCvJzmYn2WRnJYR836/Xfrn2Ws9a+1nbBV+e59lrPYoIzMzMOquspytgZma9m4PEzMxK4iAxM7OSOEjMzKwkDhIzMyuJg8TMzEriIDEzs5I4SMzMrCQOEjMzK0m/nq5Adxg6dGiMGjWqp6thZtarPPfccx9ERFVH5fpEkIwaNYqampqeroaZWa8i6U/FlHPXlpmZlcRBYmZmJXGQmJlZSfrEGImZFW/jxo3U1tby6aef9nRVrJsMGDCAESNGUFFR0an9HSRm1kptbS3bb789o0aNQlJPV8cyFhGsWrWK2tpaRo8e3aljuGvLzFr59NNPGTJkiEOkj5DEkCFDSmqBZhokkiZJWippmaQL2ihziqTFkl6RdHvO+m9Kej19fTNn/YGSXkqPeY18tZt1Of+x6ltK/f+dWdeWpHLgOuBooBZYKGluRCzOKbMXcCFwSER8JGnXdP0uwEVANRDAc+m+HwHXAzOAZ4B5wCTggUxOYukDUL8Gdt0HhuwFFQMy+Rgzs94syzGSg4BlEbEcQNKdwInA4pwy3wKuSwOCiFiZrj8WeDgiPkz3fRiYJOn3wA4R8XS6/lbgq2QVJE9fB289kSyrDHYeDbuOhaq9oWqsA8bMjGy7tnYH3sl5X5uuyzUGGCPpSUnPSJrUwb67p8vtHbPrTPt3+PbTMGUWHHY+fOYL8MFr8Ier4d/Pgl/+FVw+DK45AO44HR65BBbdDe8tgo3+xYtZZ6xevZpf/OIXndr36quvZt26dV1co65xxBFHdPoJG/fddx+LF7f8G7zYY61evZopU6awzz77MHbsWJ5++ulOfX5HsmyRFOp0iwKfvxdwBDACeELSvu3sW8wxkw+XZpB0gbHHHnsUV+N8/SrhM+OSV66GDbBqGdQtgbqlsHIJ1L0Krz0I0ZhWoAx2HtXScqlKX0PHuAVj1o6mIDn77LO3eN+rr76aadOmMWjQoAxq1nPuu+8+TjjhBMaNG9dx4Rzf+c53mDRpEnPmzGHDhg2ZhWyWQVILjMx5PwJYUaDMMxGxEXhT0lKSYKklCZfcfX+frh/RwTEBiIgbgBsAqqurC4ZNp3UYMK8mr5Vp0DhgrJe6+P5XWLziky495rjhO3DR33yhze0XXHABb7zxBhMmTODoo49m11135a677qK+vp6TTjqJiy++mLVr13LKKadQW1tLY2MjP/zhD3n//fdZsWIFRx55JEOHDuXRRx8tePzBgwdzzjnnsGDBAnbeeWcuv/xyvve97/H2229z9dVXM3nyZN566y2+8Y1vsHbtWgB+/vOf8+Uvf5l7772X6667jocffpg///nPHH744Tz++OPstttum33O+vXrOfPMM1m8eDFjx45l/fr1zdvmz5/PRRddRH19PZ///Oe56aabGDx4MKNGjeLUU09trvvtt9/OypUrmTt3Lo899hiXXXYZ99xzDwB33303Z599NqtXr+bXv/41hx56aKvP/+STT3j88ce5+eabAaisrKSysrL4/1FbIMsgWQjsJWk08C5wGnB6Xpn7gKnAzZKGknR1LQfeAC6XtHNa7hjgwoj4UNJfJB0M/BH4b8C1GZ7DltnSgHn9IdjUkJTJDZiqvdOxmH1g6F5QMbDbT8Wsp1xxxRW8/PLLvPDCC8yfP585c+bw7LPPEhFMnjyZxx9/nLq6OoYPH85//ud/AvDxxx+z4447ctVVV/Hoo48ydOjQNo+/du1ajjjiCK688kpOOukkfvCDH/Dwww+zePFivvnNbzJ58mR23XVXHn74YQYMGMDrr7/O1KlTqamp4aSTTuKee+7huuuu48EHH+Tiiy8uGCIA119/PYMGDWLRokUsWrSIAw44AIAPPviAyy67jAULFrDddttx5ZVXctVVV/GjH/0IgB122IFnn32WW2+9lb//+7/nd7/7HZMnT+aEE05gypQpzcdvaGjg2WefZd68eVx88cUsWLCAFStWcNZZZzFv3jyWL19OVVUVZ555Ji+++CIHHnggP/vZz9huu+266n9Vs8yCJCIaJM0EHgLKgVkR8YqkS4CaiJibbjtG0mKgETg/IlYBSLqUJIwALmkaeAe+DdwMDCQZZM9moL0rtRcwH77R0jVW9yqsfNUBY1uN9loO3WH+/PnMnz+f/fffH4A1a9bw+uuvc+ihh3Leeefx/e9/nxNOOGGzf423p7KykkmTkuHY/fbbj/79+1NRUcF+++3HW2+9BSR398+cOZMXXniB8vJyXnvtteb9r732Wvbdd18OPvhgpk6d2ubnPP7445x77rkAjB8/nvHjxwPwzDPPsHjxYg455BAANmzYwJe+9KXm/ZqOOXXqVL773e+2efyvfe1rABx44IHN9R4+fDjz5s0DkqB5/vnnufbaa5k4cSLf+c53uOKKK7j00kuL/q6Klemd7RExj+QnurnrfpSzHMA/pK/8fWcBswqsrwH27fLK9oR+lUkw7Dq29fpWAbM0GYtpM2DSrrGmX5MNHeOAsW1GRHDhhRfyd3/3d5tte+6555g3bx4XXnghxxxzTPO/6DtSUVHRfN9EWVkZ/fv3b15uaEj+fP30pz/lM5/5DC+++CKbNm1iwICWbud3332XsrIy3n//fTZt2kRZWdu/WSp0f0ZEcPTRR3PHHXd0uE9793c01bu8vLy53rlGjBjBiBEjmDhxIgBTpkzhiiuuaPN4pfAjUrZGWxww81sCBiUB09RyqdonGYtxwFgvsf322/OXv/wFgGOPPZYf/vCHnHHGGQwePJh3332XiooKGhoa2GWXXZg2bRqDBw9uHgdo2re9rq1ifPzxx4wYMYKysjJuueUWGhuTMc6GhgbOPPNMbr/9dm699VauuuoqzjvvvILHOOyww7jttts48sgjefnll1m0aBEABx98MOeccw7Lli1jzz33ZN26ddTW1jJmzBgAZs+ezQUXXMDs2bObWyq530mxdtttN0aOHMnSpUvZe++9eeSRR7Z4sL5YDpLepKOAaeoaa/o1mQPGeqEhQ4ZwyCGHsO+++3Lcccdx+umnN/+FOnjwYH7729+ybNkyzj//fMrKyqioqOD6668HYMaMGRx33HEMGzaszcH2Ypx99tmcfPLJ3H333Rx55JHN4wqXX345hx56KIceeigTJkzgi1/8Iscffzxjx47d7Bjf/va3OfPMMxk/fjwTJkzgoIMOAqCqqoqbb76ZqVOnUl9fD8Bll13WHCT19fVMnDiRTZs2NbdaTjvtNL71rW9xzTXXMGfOnDbrnTtGAkk33BlnnMGGDRv43Oc+x0033dTp76Q9SnqXtm3V1dXRJ2dI3Cxg0teqZW0ETM6Nlg6YPmvJkiUF/2K07DXN5lpqi6ozCv1/l/RcRFR3tK9bJNuy3BZM7phpwwb4cHlL11hTwLTZgtm7ZbB/6Bio3LZ+o29mpXGQ9EX9KpNWx677tA6Yxo2w6o3Nb7QsFDBNXWMOGNtKTZw4sbnrqMlvfvMb9ttvvy79nIceeojvf//7rdaNHj2ae++9d4uP1fTrq97GQWItyitaAiZXWwGz7OG8gPls4RstHTDWA/74xz92y+cce+yxHHvssd3yWVsrB4l1rMOAybvR0gFj1qc4SKzzig2YpsH+ZQtg08a0UE7AtLrR0gFj1ts4SKzrtRcwHy7f/E7+DgNmbxi6twPGbCvlILHuU16R/gJs79brWwVMzo2WBQMm/05+B4xZT3OQWM/b4oB5pHXA7LRHgRstHTC91erVq7n99ts7/Rj5GTNmbJWPkT/iiCP413/9V6qrO7wtYzP33XcfY8aMab4zvdhj/fSnP+XGG29EEvvttx833XRTq8e9dBUHiW29OgqY/Dv5HTDbBM9HsrnOzEfy7rvvcs0117B48WIGDhzIKaecwp133sn06dO7vH4OEut9cgNm3Ikt6zcLmPTVZsDk3clf2fWP1+71HrgA/vxS1x5zt/3guLYfHuj5SLpmPhJIng22fv16KioqWLduHcOHDy/+/9MWcJDYtqPdgHlz8zv5OwqYpmM5YLqV5yPpmvlIdt99d8477zz22GMPBg4cyDHHHMMxxxzTVf+bWnGQ2LavvAKqxiSvtgKm1Y2WBQIm/07+vhIw7bQcuoPnI+n8fCQfffQR//Ef/8Gbb77JTjvtxNe//nV++9vfMm3atKK/q2I5SKzvyg2YXI0NLc8iyw2YN/5vTsCQBkzejZZ9JWC6iecj6fx8JAsWLGD06NFUVVUBSfA89dRTmQRJ299AF5A0SdJSScskXVBg+3RJdZJeSF9n5Wy7UtLL6evUnPV/Len5tPwfJO2Z5TlYH1Ter6X1cvj34Os3wdlPwz/+Gc5ZCKf8Bo78R9i9Gj6uhWeuh/u+Db86Ei4fDlfvB7edAg//CF64Hd59Hjas7emz6jXy5yOZNWsWa9asAZK/xFeuXMmKFSsYNGgQ06ZN47zzzuP555/fbN9SfPzxxwwbNoyysjJ+85vfFJyPZOzYsVx11VVtHqNpPhJgs/lInnzySZYtWwbAunXrWrV4Zs+e3fzfUuYj2WOPPXjmmWdYt24dEcEjjzyS2VOdM2uRSCoHrgOOBmqBhZLmRsTivKKzI2Jm3r7HAwcAE4D+wGOSHoiIT4DrgRMjYomks4EfANOzOg+zZk0BUzUGmNyyvrEBPnpz8xstlz8KjRtayjW1YPLv5O8/uNtPZWvm+Ui6Zj6SiRMnMmXKFA444AD69evH/vvvz4wZMzr9nbQns/lIJH0J+KeIODZ9fyFARPxzTpnpQHWBIDkf6B8Rl6Xvfw08FBF3SVoK/LeI+GN6zO0j4n+3V5c+Ox+J9ay2AmbV6wUCpsCNlj0UMJ6PpOd4PpLN7Q68k/O+FphYoNzJkg4DXgO+GxHvAC8CF0m6ChgEHAk0tWTOAuZJWg98Ahxc6MMlzQBmQNLEM+t25f1g6F7Jq80WTM6Nlst/v1UHjFlbsgySQqNE+c2f+4E7IqJe0v8AbgGOioj5kr4IPAXUAU8DTaNJ3wW+krZIzgeuIgmX1h8UcQNwAyQtkq44IbMu0SpgchQbMDvu0XqAv+lGSwdMK56PpPtkGSS1wMic9yOAFbkFImJVzttfAVfmbPsx8GMASbcDr0uqAv5LRDRNNDAbeLDrq27WAzoKmPw7+TMMmIho9xdDvYHnIyleqUMcWQbJQmAvSaOBd4HTgNNzC0gaFhHvpW8nA0vS9eXAThGxStJ4YDwwPy23o6QxEfEayUD+kgzPwazn5QbM2L9pWd/YAB+9tfmNlm0GTO6d/G0HzIABA1i1ahVDhgzp9WFiHYsIVq1aVdIzuDILkohokDQTeAgoB2ZFxCuSLgFqImIucK6kySTdVh/S8uurCuCJ9CL+BJgWEQ0Akr4F3CNpE/AR8LdZnYPZVq28HwzdM3l1NmCq9s650TIJmxEjRlBbW0tdXV13n5H1kAEDBjBixIhO75/Zr7a2Jv7VlhmtAyb3eWQfvFZUwHgMpu/ZGn61ZWZbk2JaMM0BsxTefBwacwardxyZ86iYnMfFOGD6PAeJWV/XYcC8mtNNVmzAjIH+23f7qVjPcJCYWWGtAuaElvWNDbD6T+nPlJueR/Zq2wHTfCe/A2Zb5SAxsy1T3g+GfD55dTpg9s650bJpDMYB01s5SMysa7QVMJsaky6y/EfFvPmEA2Yb4SAxs2yVlRcfMHUFAmaHEXk3Wo51wGxlHCRm1jM6Cpi6V1uHTDEBM3QMDNih20+lr3OQmNnWJTdg9jm+Zf1mAZM+j+ytP0DDpy3lHDDdzkFiZr1DVwRM7lwwTWMwDpiSOUjMrHcrJmByH3i58EkHTBdzkJjZtmmLAubVAgGze+vB/aY7+R0wm3GQmFnf0mHA5MwFU3TAjIEBO3b7qWwtHCRmZpAXMF9pWb+pMb3RMmcumJWFush2bz0XTB8KGAeJmVl7ysphl88lr2IC5k+FAiZnLpjmMZhtJ2AcJGZmnVFUwLza8muyP/16mw2YTINE0iTgZyQTW90YEVfkbZ8O/AvJDIoAP4+IG9NtVwJNHZiXRsTsdL2Ay4CvA43A9RFxTZbnYWZWtI4Cpqnl0hQyNbOgYX1Lue2H53SN7d0yFrMVB0xmQZJOl3sdyXS4tcBCSXMjYnFe0dkRMTNv3+OBA4AJQH/gMUkPRMQnJLMojgT2iYhNknbN6hzMzLpMbsDsfVzL+i0OmLxHxWwFAZNli+QgYFlELAeQdCdwIpAfJIWMAx5Lp9dtkPQiMAm4C/g2cHpEbAKIiJVZVN7MrFu0GzBvb36jZc1NW13AZBkkuwPv5LyvBSYWKHeypMOA14DvRsQ7wIvARZKuAgYBR9ISQJ8HTpV0ElAHnBsRr2d0DmZmPaOsHHYZnbw6GzBVe8MJVyUhlaEsg0QF1uVPEH8/cEdE1Ev6H8AtwFERMV/SF4GnSMLiaaAh3ac/8GlEVEv6GjALOHSzD5dmADMA9thjj644HzOzntdmwGxKu8jy7uSvzP4pyYrI/7u9iw4sfQn4p4g4Nn1/IUBE/HMb5cuBDyNis/aYpNuB30bEPEmvApMi4q104H11oX1yVVdXR01NTYlnZGbWt0h6LiKqOypXlmEdFgJ7SRotqRI4DZibW0DSsJy3k4El6fpySUPS5fHAeGB+Wu4+4Kh0+XCSLjEzM+shmXVtRUSDpJnAQyQ//50VEa9IugSoiYi5wLmSJpN0W31I8ossgArgiaTBwSfAtHTgHeAK4DZJ3wXWAGdldQ5mZtaxzLq2tibu2jIz23JbQ9eWmZn1AQ4SMzMriYPEzMxK4iAxM7OSOEjMzKwkDhIzMyuJg8TMzEriIDEzs5I4SMzMrCQOEjMzK4mDxMzMSuIgMTOzkjhIzMysJA4SMzMriYPEzMxK4iAxM7OSZBokkiZJWippmaQLCmyfLqlO0gvp66ycbVdKejl9nVpg32slrcmy/mZm1rHMptqVVA5cBxwN1AILJc2NiMV5RWdHxMy8fY8HDgAmAP2BxyQ9EBGfpNurgZ2yqruZmRUvyxbJQcCyiFgeERuAO4ETi9x3HPBYRDRExFrgRWASNAfUvwDfy6DOZma2hbIMkt2Bd3Le16br8p0saZGkOZJGputeBI6TNEjSUOBIoGnbTGBuRLyXVcXNzKx4WQaJCqyLvPf3A6MiYjywALgFICLmA/OAp4A7gKeBBknDga8D13b44dIMSTWSaurq6jp/FmZm1q4sg6SWllYEwAhgRW6BiFgVEfXp218BB+Zs+3FETIiIo0lC6XVgf2BPYJmkt4BBkpYV+vCIuCEiqiOiuqqqqqvOyczM8mQ22A4sBPaSNBp4FzgNOD23gKRhOV1Uk4El6fpyYKeIWCVpPDAemB8RDcBuOfuviYg9MzwHMzPrQIdBIunzQG1E1Es6guQv9VsjYnV7+0VEg6SZwENAOTArIl6RdAlQExFzgXMlTQYagA+B6enuFcATkgA+AaalIWJmZlsZReQPW+QVkF4AqoFRJKEwF9g7Ir6See26SHV1ddTU1PR0NczMehVJz0VEdUflihkj2ZS2Bk4Cro6I7wLDSq2gmZltG4oJko2SpgLfBH6XrqvIrkpmZtabFBMkZwJfAn4cEW+mg+e/zbZaZmbWW3Q42J4+0uRcAEk7A9tHxBVZV8zMzHqHDlskkn4vaQdJu5DccX6TpKuyr5qZmfUGxXRt7Zg+LPFrwE0RcSDwX7OtlpmZ9RbFBEk/ScOAU2gZbDczMwOKC5JLSO4feSMiFkr6HMnjSszMzIoabL8buDvn/XLg5CwrZWZmvUcxg+0jJN0raaWk9yXdI2lEd1TOzMy2fsV0bd1E8liU4STzidyfrjMzMysqSKoi4qZ0tsKGiLgZ8HPZzcwMKC5IPpA0TVJ5+poGrMq6YmZm1jsUEyR/S/LT3z8D7wFTSB6bYmZm1nGQRMTbETE5IqoiYteI+CrJzYlmZmadnmr3H7q0FmZm1mt1NkhUVCFpkqSlkpZJuqDA9umS6iS9kL7Oytl2paSX09epOetvS4/5sqRZkvxIezOzHtTZIGl/WkWa512/DjgOGAdMlTSuQNHZETEhfd2Y7ns8cAAwAZgInC9ph7T8bcA+wH7AQOCsAsc0M7Nu0uad7ZL+QuHAEMlf4B05CFiW3gmPpDuBE4HFRew7DngsnZmxQdKLwCTgroiYl1PHZwHfHGlm1oPabJFExPYRsUOB1/YR0eGjVUhuXnwn531tui7fyZIWSZojaWS67kXgOEmDJA0FjgRG5u6Udml9A3iwiLqYmVlGOtu1VYxC4yj5LZz7gVERMR5YANwCEBHzgXnAU8AdwNNAQ96+vwAej4gnCn64NENSjaSaurq6zp+FmZm1K8sgqaV1K2IEsCK3QESsioj69O2vgANztv04HTc5miSUmp84LOkikrvr2/z1WETcEBHVEVFdVeUb8c3MspJlkCwE9pI0WlIlcBrJM7uapfOcNJkMLEnXl0saki6PB8YD89P3ZwHHAlMjYlOG9TczsyIUM9bRKRHRIGkmyVwm5cCsiHhF0iVATUTMBc6VNJmk2+pDYHq6ewXwhCSAT4Bp6cA7wC+BPwFPp9v/PSIuyeo8zMysfYpo/5e8bfx662OgBvhfTb/K2ppVV1dHTU1NT1fDzKxXkfRcRFR3VK6YFslVJGMbt5OMVZwG7AYsBWYBR3S+mmZm1tsVM0YyKSL+LSL+EhGfRMQNwFciYjawc8b1MzOzrVwxQbJJ0imSytLXKTnbOrzD3czMtm3FBMkZJDf+rUxf3wCmSRoIzMywbmZm1gt0OEaSDqb/TRub/9C11TEzs96mwxaJpBGS7pW0UtL7ku6R5OdbmZkZUFzX1k0kNxIOJ3lW1v3pOjMzs6KCpCoiboqIhvR1M8njSczMzIoKkg8kTUsfW1IuaRqwKuuKmZlZ71BMkPwtcArwZ+A9YApwZpaVMjOz3qPDIImItyNickRURcSuEfFV4GvdUDczM+sFOvv03zYf325mZn1LZ4Ok0KRVZmbWB3U2SPxoFDMzA9q5s72Nx8dD0hoZmFmNzMysV2kzSCJi++6siJmZ9U5ZTrWLpEmSlkpaJumCAtunS6qT9EL6Oitn25WSXk5fp+asHy3pj5JelzQ7ncbXzMx6SGZBIqkcuA44DhgHTJU0rkDR2RExIX3dmO57PHAAMAGYCJwvaYe0/JXATyNiL+Aj4L9ndQ5mZtaxLFskBwHLImJ5RGwA7gROLHLfccBj6SNZ1gIvApOUTNJ+FDAnLXcL8NUurreZmW2BLINkd+CdnPe16bp8J0taJGmOpJHpuheB4yQNkjQUOBIYCQwBVkdEQwfHRNIMSTWSaurq6rrifMzMrIAsg6TQvSb5vwK7HxgVEeOBBSQtDCJiPjAPeAq4A3gaaCjymKTHuCEiqiOiuqrKz5g0M8tKlkFSS9KKaDICWJFbICJWRUR9+vZXwIE5236cjpscTRIgrwMfADtJ6tfWMc3MrHtlGSQLgb3SX1lVAqeRzGvSTNKwnLeTgSXp+nJJQ9Ll8cB4YH5EBPAoyYMjAb4J/EeG52BmZh3ocKrdzoqIBkkzgYeAcmBWRLwi6RKgJiLmAudKmkzSbfUhMD3dvQJ4Ihlb5xNgWs64yPeBOyVdBvw/4NdZnYOZmXVMyT/yt23V1dVRU1PT09UwM+tVJD0XEdUdlcv0hkQzM9v2OUjMzKwkDhIzMyuJg8TMzEriIDEzs5I4SMzMrCQOEjMzK4mDxMzMSuIgMTOzkjhIzMysJA4SMzMriYPEzMxK4iAxM7OSOEjMzKwkDhIzMyuJg8TMzEqSaZBImiRpqaRlki4osH26pDpJL6Svs3K2/UTSK5KWSLpG6XSJkqZKeknSIkkPShqa5TmYmVn7MgsSSeXAdcBxwDhgqqRxBYrOjogJ6evGdN8vA4eQzNW+L/BF4HBJ/YCfAUdGxHhgETAzq3MwM7OOZdkiOQhYFhHLI2IDcCdwYpH7BjAAqAT6k8zh/j6g9LVd2kLZAVjR1RU3M7PiZRkkuwPv5LyvTdflOzntppojaSRARDwNPAq8l74eioglEbER+DbwEkmAjAN+XejDJc2QVCOppq6urstOyszMWssySFRgXeS9vx8YlXZTLQBuAZC0JzAWGEESPkdJOkxSBUmQ7A8MJ+naurDQh0fEDRFRHRHVVVVVXXE+ZmZWQJZBUguMzHk/grxuqIhYFRH16dtfAQemyycBz0TEmohYAzwAHAxMSPd7IyICuAv4cnanYGZmHckySBYCe0kaLakSOA2Ym1tA0rCct5OBJeny26SD62kr5PB027vAOElNTYyjc/YxM7Me0C+rA0dEg6SZwENAOTArIl6RdAlQExFzgXMlTQYagA+B6enuc4CjSMZCAngwIu4HkHQx8LikjcCfcvYxM7MeoKSHaNtWXV0dNTU1PV0NM7NeRdJzEVHdUTnf2W5mZiVxkJiZWUkcJGZmVhIHiZmZlcRBYmZmJXGQmJlZSRwkZmZWEgeJmZmVxEFiZmYlcZCYmVlJHCRmZlYSB4mZmZXEQWJmZiVxkJiZWUkcJGZmVpJMg0TSJElLJS2TdEGB7dMl1Ul6IX2dlbPtJ5JekbRE0jWSlK6vlHSDpNckvSrp5CzPwczM2pfZDImSyoHrSKbDrQUWSpobEYvzis6OiJl5+34ZOAQYn676A8l0u78H/hFYGRFjJJUBu2R1DmZm1rHMggQ4CFgWEcsBJN0JnAjkB0khAQwAKgEBFcD76ba/BfYBiIhNwAddW20zM9sSWXZt7Q68k/O+Nl2X72RJiyTNkTQSICKeBh4F3ktfD0XEEkk7pftcKul5SXdL+kyG52BmZh3IMkhUYF3+BPH3A6NCMRKmAAAOV0lEQVQiYjywALgFQNKewFhgBEn4HCXpMJIW1AjgyYg4AHga+NeCHy7NkFQjqaaurq4rzsfMzArIMkhqgZE570cAK3ILRMSqiKhP3/4KODBdPgl4JiLWRMQa4AHgYGAVsA64Ny13N3BAoQ+PiBsiojoiqquqqrrifMzMrIAsg2QhsJek0ZIqgdOAubkFJA3LeTsZWJIuvw0cLqmfpAqSgfYlEREkrZgj0nJ/TXFjLmZmlpHMBtsjokHSTOAhoByYFRGvSLoEqImIucC5kiYDDcCHwPR09znAUcBLJN1hD0bE/em27wO/kXQ1UAecmdU5mJlZx5T8I3/bVl1dHTU1NT1dDTOzXkXScxFR3VE539luZmYlcZCYmVlJHCRmZlaSLO9s7/XeXrWODY2NDKrsx6DKcgZWllNZXkb62C8zM8NB0q6L5r7Mo0tb38zYr0wMrCxnUGU521X2a14eWNmP7dKwyd/WFESbl2u9XF7mgDKz3sdB0o6ZR+3JV/ffnfUbGlm3oZF1GxrS/7Ysr9/QyNoNDXy8bgPvrW69rb5h0xZ9Xv9+Zc3BM7CyvFXIDGojlAqtz21BbVfZjwEVbkWZWXYcJO048LO7cOBnO79/46Zg3YaG5iBam7NcKIwKlVu/oZH3P/l0s20Nm4r/2bYEAyvyw6YlsDoKokGV5W2Wq+znYTazvs5BkqHyMrH9gAq2H1DR5cfe0LCpVQupvTBqXa6BtWn5dRsa+GBNfatgW7+xkS25taipq6+twMnf1rrLr73QclefWW/hIOmlKvuVUdmvkp0Gde1xI4JPN24q2I1XKJTaKrN63QbeXd0SWKV29bV053Vu/Ck3uNzVZ9a1HCTWipS0MAZWljOki4/d0LiJ9RubWk+Nzd1+a9sIpaZWVstyUu7PaVff2px9Grewq29QRf4YU+Hxp+06GIvK/3GFu/qsL3KQWLfpV17G9uVlXd7VFxFsaNzURldefhi1P07V0tXXcowtOscytQ6c/uUMqihiLKp/OQMr2gis/v0YWFHurj7bajlIrNeTRP9+5fTvV55ZV19nxp9ylz/K6+pbu6GRDZ3o6tsuDZW2xpiaW0g55TYbp2p1DHf1WekcJGbtyO3q62pNXX3tjT+11e23rr6RdRubuvo2brZ/Z7v6kpZRoTBKxpyal5vL9UtbXTk/W89phbmrr29wkJj1kO7o6msziDY0sq6+IQ2jRtbWN7J+Y+tya+obqPtL/WbdgVuiolytA6eyJWRadeel4dO8XNmybbNyle7q29o4SMy2MVl29W3aFHza0Mb4U31DcwtrbX0aPhtbtuUuN3X1NYXZuk509Q2oKGsOlaaxpEF5y80tpPbKVSatq+3S5f793NW3pRwkZla0sjKl4zJd/1dHQ+Om5rBZlxdM6+rT1lLanZe0oNIWVn1j87Z19Q289/HG1ts2bllXX1nTDbwFQqZpeWBleZth1LSteTlthQ3qX05F+bbZ1ZdpkEiaBPyMZIbEGyPiirzt04F/Ad5NV/08Im5Mt/0EOJ7kCcUPA9+JnFm4JM0FPhcR+2Z5DmbWPfqVl7FDeRk7ZNTVlzuutDYNn+buvPo0eDY2NgfT+o2ty62pb2DlJ/Ws29iQ0x3Yua6+ViGT+7PzinT8KQ2f5uX2ylUmLayyHuzqyyxIJJUD1wFHA7XAQklzIyJ/jvXZETEzb98vA4cA49NVfyCZt/336favAWuyqruZbTtyu/p27uJj53b1JUHVOpjWb2xMw6ilqy+3ddW0/OHaDbzzYU6ZErr6Wt0LVVHOT6aMZ+QuXdzHmSfLFslBwLKIWA4g6U7gRCA/SAoJYABQCQioAN5PjzMY+AdgBnBX11fbzKw4rbr6BnftsXO7+tamgZMEU86PH3LCKH9b048k+pVn31LJMkh2B97JeV8LTCxQ7mRJhwGvAd+NiHci4mlJjwLvkQTJzyNiSVr+UuD/AOuyq7qZWc/KqqsvC1mO/BSKwfwRr/uBURExHlgA3AIgaU9gLDCCJJCOknSYpAnAnhFxb4cfLs2QVCOppq6urqPiZmbWSVkGSS0wMuf9CGBFboGIWBUR9enbXwEHpssnAc9ExJqIWAM8ABwMfAk4UNJbJOMmYyT9vtCHR8QNEVEdEdVVVVVddEpmZpYvyyBZCOwlabSkSuA0YG5uAUnDct5OBpq6r94GDpfUT1IFyUD7koi4PiKGR8Qo4K+A1yLiiAzPwczMOpDZGElENEiaCTxE8vPfWRHxiqRLgJqImAucK2ky0AB8CExPd58DHAW8RNId9mBE3J9VXc3MrPMUWzKLUS9VXV0dNTU1PV0NM7NeRdJzEVHdUblt8zZLMzPrNg4SMzMriYPEzMxK0ifGSCTVAX/q5O5DgQ+6sDpdxfXaMq7XlnG9tsy2Wq/PRkSH90/0iSAphaSaYgabupvrtWVcry3jem2Zvl4vd22ZmVlJHCRmZlYSB0nHbujpCrTB9doyrteWcb22TJ+ul8dIzMysJG6RmJlZSfpskEiaJWmlpJfb2C5J10haJmmRpANytn1T0uvp65vdXK8z0voskvSUpP+Ss+0tSS9JekFSlz4Tpoh6HSHp4/SzX5D0o5xtkyQtTb/LC7q5Xufn1OllSY2Sdkm3Zfl9jZT0qKQlkl6R9J0CZbr9GiuyXt1+jRVZr26/xoqsV7dfY5IGSHpW0otpvS4uUKa/pNnpd/JHSaNytl2Yrl8q6diSKxQRffIFHAYcALzcxvavkDy+XiSPsP9jun4XYHn6353T5Z27sV5fbvo84LimeqXv3wKG9tD3dQTwuwLry4E3gM+RzHj5IjCuu+qVV/ZvgP/bTd/XMOCAdHl7konbxuWV6fZrrMh6dfs1VmS9uv0aK6ZePXGNpdfM4HS5AvgjcHBembOBX6bLp5FMaw4wLv2O+gOj0++uvJT69NkWSUQ8TvLE4bacCNwaiWeAnZQ89v5Y4OGI+DAiPgIeBiZ1V70i4qn0cwGeIZnnJXNFfF9taZ5yOSI2AE1TLvdEvaYCd3TVZ7cnIt6LiOfT5b+QTJGwe16xbr/GiqlXT1xjRX5fbcnsGutEvbrlGkuvmTXp24r0lT/gfSLpZIEkT1T/a0lK198ZEfUR8SawjOQ77LQ+GyRFKDRV8O7trO8J/53kX7RNApgv6TlJM3qgPl9Km9oPSPpCum6r+L4kDSL5y/ienNXd8n2lXQr7k/yrMVePXmPt1CtXt19jHdSrx66xjr6v7r7GJJVLegFYSfIPjzavr4hoAD4GhpDB95XlnO29XVtTBRczhXDmJB1J8of8r3JWHxIRKyTtCjws6dX0X+zd4XmSxymskfQV4D5gL7aS74uky+HJiMhtvWT+fUkaTPIXy99HxCf5mwvs0i3XWAf1airT7ddYB/XqsWusmO+Lbr7GIqIRmCBpJ+BeSftGRO5YYbddX26RtK2tqYI7nEI4a5LGAzcCJ0bEqqb1EbEi/e9K4F5KbK5uiYj4pKmpHRHzgApJQ9kKvq/UaeR1OWT9fSmZ3fMe4LaI+PcCRXrkGiuiXj1yjXVUr566xor5vlLdfo2lx14N/J7Nuz+bvxdJ/YAdSbqBu/776soBoN72AkbR9uDx8bQeCH02Xb8L8CbJIOjO6fIu3VivPUj6NL+ct347YPuc5aeASd1Yr91ouS/pIJLpkkXS6l1OMqjXNBD6he6qV7q96Q/Qdt31faXnfitwdTtluv0aK7Je3X6NFVmvbr/GiqlXT1xjQBWwU7o8EHgCOCGvzDm0Hmy/K13+Aq0H25dT4mB7n+3aknQHya9AhkqqBS4iGbAiIn4JzCP5Vc0yYB1wZrrtQ0mXksxJD3BJtG7KZl2vH5H0c/4iGTejIZKHsn2GpHkLyR+s2yPiwW6s1xTg25IagPXAaZFctQWnXO7GegGcBMyPiLU5u2b6fQGHAN8AXkr7sQH+N8lf0j15jRVTr564xoqpV09cY8XUC7r/GhsG3CKpnKRn6a6I+J1aT2X+a+A3kpaRhNxpaZ1fkXQXsJhkmvNzIukm6zTf2W5mZiXxGImZmZXEQWJmZiVxkJiZWUkcJGZmVhIHiZmZlcRBYmZmJXGQmG0l0keOD+3kvtMlDe+KY5ltKQeJ2bZhOjC8o0JmWXCQmOWRNErSq5JuTCcquk3Sf5X0pJKJpg5KX09J+n/pf/dO9/0HSbPS5f3S/Qe18TlDJM1Pj/Fv5DxMT9K0dOKiFyT9W3oHM5LWSPo/kp6X9IikKklTgGrgtrT8wPQw/zMt95KkfbL8zqxvc5CYFbYn8DNgPLAPcDrJU3DPI3lExqvAYRGxP8kjRS5P97sa2FPSScBNwN9FxLo2PuMi4A/pMeaSPnZD0ljgVJInx04AGoEz0n22A56PiAOAx4CLImIOUAOcERETImJ9WvaDtNz1ab3NMtFnn7Vl1oE3I+IlAEmvAI9EREh6ieQhkTuSPOtoL5JHcDc932uTpOnAIuDfIuLJdj7jMOBr6X7/KalpMqm/Bg4EFqbPaRpIMucEwCZgdrr8W6C9p9E2bXuu6XPMsuAgMSusPmd5U877TSR/bi4FHo2Ik9IJj36fU34vYA3FjVkUetidgFsi4sJO7t+kqc6N+M+6ZchdW2adsyPwbro8vWmlpB1JusQOA4ak4xdteZy0y0rScSSPjAd4BJiSToaEpF0kfTbdVkbyFFxIutv+kC7/hWROcbNu5yAx65yfAP8s6UmSR5c3+Snwi4h4jWR2wSuaAqGAi4HDJD0PHEMyvwYRsRj4AckUrYtI5mwflu6zFviCpOeAo4BL0vU3A7/MG2w36xZ+jLxZLyJpTUQM7ul6mOVyi8TMzEriFolZxiSdCXwnb/WTEXFOT9THrKs5SMzMrCTu2jIzs5I4SMzMrCQOEjMzK4mDxMzMSuIgMTOzkvx/Sn2RDJ8FgSgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a79a198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2-2.进一步细调max_depth&min_child_weigh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 粗调得出的max_depth=6, min_child_weight=3，在其邻域再寻找更佳的参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [5, 6, 7], 'min_child_weight': [2, 3, 4]}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = [5,6,7]\n",
    "min_child_weight = [2,3,4]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58384, std: 0.00353, params: {'max_depth': 5, 'min_child_weight': 2},\n",
       "  mean: -0.58388, std: 0.00346, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58400, std: 0.00356, params: {'max_depth': 5, 'min_child_weight': 4},\n",
       "  mean: -0.58468, std: 0.00365, params: {'max_depth': 6, 'min_child_weight': 2},\n",
       "  mean: -0.58414, std: 0.00375, params: {'max_depth': 6, 'min_child_weight': 3},\n",
       "  mean: -0.58409, std: 0.00316, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.58969, std: 0.00406, params: {'max_depth': 7, 'min_child_weight': 2},\n",
       "  mean: -0.58726, std: 0.00434, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58815, std: 0.00376, params: {'max_depth': 7, 'min_child_weight': 4}],\n",
       " {'max_depth': 5, 'min_child_weight': 2},\n",
       " -0.5838375452658044)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=338,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,      #第2-1轮参数调整得到的max_depth最优值\n",
    "        min_child_weight=3,#第2-1轮参数调整得到的min_child_weight最优值\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 得出 max_depth=5, min_child_weight=2,logloss为0.5838"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.调整正则参数reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.1, 1.5, 2], 'reg_lambda': [0.1, 0.5, 1, 2]}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [0.1, 1.5, 2]    #default = 0, 测试0.1,1.5，2\n",
    "reg_lambda = [0.1,0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test3 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58393, std: 0.00388, params: {'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58379, std: 0.00386, params: {'reg_alpha': 0.1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58367, std: 0.00443, params: {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  mean: -0.58334, std: 0.00380, params: {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  mean: -0.58385, std: 0.00379, params: {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       "  mean: -0.58344, std: 0.00402, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58367, std: 0.00405, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58367, std: 0.00401, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58330, std: 0.00389, params: {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  mean: -0.58417, std: 0.00363, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58341, std: 0.00373, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58369, std: 0.00393, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       " -0.5832964635140194)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=338,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,       #第2-2轮参数调整得到的max_depth最优值\n",
    "        min_child_weight=2,#第2-2轮参数调整得到的min_child_weight最优值\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3 = GridSearchCV(xgb3, param_grid = param_test3, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3.fit(x_train , y_train)\n",
    "\n",
    "gsearch3.grid_scores_, gsearch3.best_params_,     gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### reg_alpha=2 reg_lambda=0.1，logloss为0.5832"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.重新调整弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train : 0.4940031071537599\n"
     ]
    }
   ],
   "source": [
    "xgb4 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #重新设置为1000\n",
    "        max_depth=5,       #第2-2轮参数调整得到的max_depth最优值\n",
    "        min_child_weight=2,#第2-2轮参数调整得到的min_child_weight最优值\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        reg_alpha = 2, #第三轮参数调整得到的reg_alpha最优值\n",
    "        reg_lambda =0.1,  #第三轮参数调整得到的reg_lambda最优值\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb4, x_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 0.7,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.1,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 5,\n",
       " 'min_child_weight': 2,\n",
       " 'missing': None,\n",
       " 'n_estimators': 338,\n",
       " 'nthread': 1,\n",
       " 'objective': 'multi:softprob',\n",
       " 'reg_alpha': 2,\n",
       " 'reg_lambda': 0.1,\n",
       " 'scale_pos_weight': 1,\n",
       " 'seed': 3,\n",
       " 'silent': 1,\n",
       " 'subsample': 0.5}"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb4.get_xgb_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 重新调整后，弱学习器数目为338，loglogss0.4940"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.行列重采样参数subsample 和 colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.6, 0.7, 0.8, 0.9],\n",
       " 'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test5 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58826, std: 0.00320, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.58497, std: 0.00381, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58336, std: 0.00272, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58230, std: 0.00369, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58093, std: 0.00334, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58076, std: 0.00245, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58701, std: 0.00448, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58445, std: 0.00381, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58256, std: 0.00420, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58242, std: 0.00332, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58106, std: 0.00378, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58001, std: 0.00406, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58662, std: 0.00401, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58599, std: 0.00352, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58330, std: 0.00389, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58297, std: 0.00360, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58159, std: 0.00364, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58002, std: 0.00390, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58664, std: 0.00321, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58509, std: 0.00349, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58315, std: 0.00371, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58207, std: 0.00363, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58094, std: 0.00330, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58036, std: 0.00343, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       " -0.5800092325414781)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=338,  #第四轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,       #第2-2轮参数调整得到的max_depth最优值\n",
    "        min_child_weight=2,#第2-2轮参数调整得到的min_child_weight最优值\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        reg_alpha = 2, #第三轮参数调整得到的reg_alpha最优值\n",
    "        reg_lambda =0.1,  #第三轮参数调整得到的reg_lambda最优值\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5 = GridSearchCV(xgb5, param_grid = param_test5, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5.fit(x_train , y_train)\n",
    "\n",
    "gsearch5.grid_scores_, gsearch5.best_params_,     gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 得出colsample_bytree': 0.7, 'subsample': 0.8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6.最终的训练好参数的模型进行测试,保存测试结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb6 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=338,  #第四轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,       #第2-2轮参数调整得到的max_depth最优值\n",
    "        min_child_weight=2,#第2-2轮参数调整得到的min_child_weight最优值\n",
    "        gamma=0,\n",
    "        subsample=0.8,      #第五轮参数调整得到的reg_alpha最优值\n",
    "        colsample_bytree=0.7,#第五轮参数调整得到的reg_alpha最优值\n",
    "        colsample_bylevel = 0.7,\n",
    "        reg_alpha = 2, #第三轮参数调整得到的reg_alpha最优值\n",
    "        reg_lambda =0.1,  #第三轮参数调整得到的reg_lambda最优值\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "xgb6.fit(x_train,y_train)\n",
    "test_predprob = xgb6.predict_proba(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.11714378, 0.43120295, 0.4516533 ],\n",
       "       [0.26404187, 0.44902414, 0.28693393],\n",
       "       [0.0509216 , 0.1167876 , 0.83229077],\n",
       "       ...,\n",
       "       [0.05039321, 0.3062598 , 0.643347  ],\n",
       "       [0.48313648, 0.4111994 , 0.10566419],\n",
       "       [0.04810936, 0.35549906, 0.5963916 ]], dtype=float32)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_predprob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(test_predprob).to_csv('y_pred_test.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
